# -*- coding: utf-8 -*-
import time
from scrapy_redis.spiders import RedisSpider
from bs4 import BeautifulSoup
from ..items import NewssiteItem
import re


class NeteaseNewsSpider(RedisSpider):
    name = 'neteaseNews'
    allowed_domains = ['163.com']
    custom_settings = {
        'ITEM_PIPELINES': {
            'news_spiders.pipelines.NewssitePipeline': 200,
        }
    }

    def parse(self, response):
        soup = BeautifulSoup(response.body, 'lxml', from_encoding='utf-8')
        item = NewssiteItem()
        title = soup.select_one('.post_title')
        if title is None:
            print('[网页格式错误]')
            return
        item['title'] = title.text

        item['url'] = response.url

        # 获取正文内容
        content = soup.select('.post_body')

        item['content'] = ''
        for line in content:
            item['content'] += line.text.strip().replace(u'\u3000',
                                                         ' ').replace(u'\xa0', ' ')
        # print(item['content'])

        # 原标题
        # firstLine = content[0].text.lstrip().replace(
        #     u'\u3000', '').replace(u'\xa0', '')
        # if firstLine.startswith('（原标题：'):
        #     item['sourceTitle'] = firstLine[5:-1]

        # 来源 日期
        post_info = soup.select_one('.post_info')
        if post_info is not None:
            info = post_info.text.strip().replace('\n', '')
            pattern = re.compile(r'(?<=来源:).*?(?=举报|$)')

            item['source'] = pattern.search(info).group() if pattern.search(info) else None
            item['date'] = time.strftime(
                '%Y-%m-%d %H:%M:%S', time.strptime(info[0:19], r'%Y-%m-%d %H:%M:%S'))

        yield item
        return
